{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import email_read_util"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download 2007 TREC Public Spam Corpus\n",
    "1. Read the \"Agreement for use\"\n",
    "   https://plg.uwaterloo.ca/~gvcormac/treccorpus07/\n",
    "\n",
    "2. Download 255 MB Corpus (trec07p.tgz) and untar into the 'chapter1/datasets' directory\n",
    "\n",
    "3. Check that the below paths for 'DATA_DIR' and 'LABELS_FILE' exist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = 'datasets/trec07p/data/'\n",
    "LABELS_FILE = 'datasets/trec07p/full/index'\n",
    "TRAINING_SET_RATIO = 0.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = {}\n",
    "spam_words = set()\n",
    "ham_words = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the labels\n",
    "with open(LABELS_FILE) as f:\n",
    "    for line in f:\n",
    "        line = line.strip()\n",
    "        label, key = line.split()\n",
    "        labels[key.split('/')[-1]] = 1 if label.lower() == 'ham' else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split corpus into train and test sets\n",
    "filelist = os.listdir(DATA_DIR)\n",
    "X_train = filelist[:int(len(filelist)*TRAINING_SET_RATIO)]\n",
    "X_test = filelist[int(len(filelist)*TRAINING_SET_RATIO):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Blacklist of 97939 tokens successfully built/loaded\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists('blacklist.pkl'):\n",
    "    for filename in X_train:\n",
    "        path = os.path.join(DATA_DIR, filename)\n",
    "        if filename in labels:\n",
    "            label = labels[filename]\n",
    "            stems = email_read_util.load(path)\n",
    "            if not stems:\n",
    "                continue\n",
    "            if label == 1:\n",
    "                ham_words.update(stems)\n",
    "            elif label == 0:\n",
    "                spam_words.update(stems)\n",
    "            else:\n",
    "                continue\n",
    "    blacklist = spam_words - ham_words\n",
    "    pickle.dump(blacklist, open('blacklist.pkl', 'wb'))\n",
    "else:\n",
    "    blacklist = pickle.load(open('blacklist.pkl', 'rb') )\n",
    "\n",
    "print('Blacklist of {} tokens successfully built/loaded'.format(len(blacklist)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'pectora',\n",
       " 'sleet',\n",
       " 'soma',\n",
       " 'sorb',\n",
       " 'raglan',\n",
       " 'pluma',\n",
       " 'thrower',\n",
       " 'ducal',\n",
       " 'vatman',\n",
       " 'biaxial',\n",
       " 'choral',\n",
       " 'muzz',\n",
       " 'merk',\n",
       " 'degum',\n",
       " 'lino',\n",
       " 'punctual',\n",
       " 'zig',\n",
       " 'whopper',\n",
       " 'saunter',\n",
       " 'commot',\n",
       " 'pian',\n",
       " 'enchant',\n",
       " 'starlit',\n",
       " 'handmaid',\n",
       " 'matchbook',\n",
       " 'chil',\n",
       " 'prote',\n",
       " 'cush',\n",
       " 'feme',\n",
       " 'cerulean',\n",
       " 'flamenco',\n",
       " 'fie',\n",
       " 'adroit',\n",
       " 'calor',\n",
       " 'electrician',\n",
       " 'batik',\n",
       " 'depositor',\n",
       " 'ammonia',\n",
       " 'dor',\n",
       " 'humph',\n",
       " 'throb',\n",
       " 'osteosarcoma',\n",
       " 'phylum',\n",
       " 'staphylococci',\n",
       " 'redub',\n",
       " 'romaunt',\n",
       " 'castrum',\n",
       " 'billyboy',\n",
       " 'snug',\n",
       " 'phosphor',\n",
       " 'caroli',\n",
       " 'sweatband',\n",
       " 'sequent',\n",
       " 'flutter',\n",
       " 'abac',\n",
       " 'subjunct',\n",
       " 'matriarch',\n",
       " 'trisect',\n",
       " 'hazelwood',\n",
       " 'crept',\n",
       " 'sienna',\n",
       " 'camelopard',\n",
       " 'austral',\n",
       " 'amaranth',\n",
       " 'embank',\n",
       " 'briar',\n",
       " 'fishtail',\n",
       " 'shou',\n",
       " 'chessboard',\n",
       " 'reman',\n",
       " 'ponent',\n",
       " 'coxa',\n",
       " 'ornithologist',\n",
       " 'colter',\n",
       " 'ammono',\n",
       " 'bajada',\n",
       " 'heterotopia',\n",
       " 'cand',\n",
       " 'rainless',\n",
       " 'nar',\n",
       " 'polka',\n",
       " 'burrow',\n",
       " 'augend',\n",
       " 'irrupt',\n",
       " 'quarrel',\n",
       " 'nothing',\n",
       " 'goldfish',\n",
       " 'fibrin',\n",
       " 'anvil',\n",
       " 'bree',\n",
       " 'malm',\n",
       " 'coloss',\n",
       " 'tenderloin',\n",
       " 'siskin',\n",
       " 'precinct',\n",
       " 'sider',\n",
       " 'centrum',\n",
       " 'rhymer',\n",
       " 'artificial',\n",
       " 'brackish',\n",
       " 'schooner',\n",
       " 'quickset',\n",
       " 'repugn',\n",
       " 'ironwork',\n",
       " 'chare',\n",
       " 'sparerib',\n",
       " 'thee',\n",
       " 'citril',\n",
       " 'stade',\n",
       " 'facer',\n",
       " 'escolar',\n",
       " 'cyp',\n",
       " 'orograph',\n",
       " 'fennel',\n",
       " 'saltbush',\n",
       " 'cantar',\n",
       " 'bedpost',\n",
       " 'conic',\n",
       " 'unal',\n",
       " 'retrogress',\n",
       " 'baw',\n",
       " 'yuca',\n",
       " 'humanist',\n",
       " 'lour',\n",
       " 'jacinth',\n",
       " 'jud',\n",
       " 'pergola',\n",
       " 'simular',\n",
       " 'passeriform',\n",
       " 'protean',\n",
       " 'cashier',\n",
       " 'mejorana',\n",
       " 'affront',\n",
       " 'dragoon',\n",
       " 'expansion',\n",
       " 'moksha',\n",
       " 'smolt',\n",
       " 'poter',\n",
       " 'baboon',\n",
       " 'pater',\n",
       " 'elegiac',\n",
       " 'mellow',\n",
       " 'gopher',\n",
       " 'hiation',\n",
       " 'boomerang',\n",
       " 'birk',\n",
       " 'conclusion',\n",
       " 'credenza',\n",
       " 'gat',\n",
       " 'maud',\n",
       " 'fana',\n",
       " 'nativist',\n",
       " 'astern',\n",
       " 'karma',\n",
       " 'garlic',\n",
       " 'cinerea',\n",
       " 'individualist',\n",
       " 'saddler',\n",
       " 'nonet',\n",
       " 'tomahawk',\n",
       " 'petulant',\n",
       " 'slut',\n",
       " 'bronchi',\n",
       " 'precess',\n",
       " 'lank',\n",
       " 'raffia',\n",
       " 'anonymous',\n",
       " 'toother',\n",
       " 'condominium',\n",
       " 'beechwood',\n",
       " 'mermaid',\n",
       " 'macer',\n",
       " 'abut',\n",
       " 'hereabout',\n",
       " 'autobahn',\n",
       " 'wort',\n",
       " 'bolling',\n",
       " 'kelt',\n",
       " 'eland',\n",
       " 'excreta',\n",
       " 'appet',\n",
       " 'cowbird',\n",
       " 'shama',\n",
       " 'spart',\n",
       " 'ordinar',\n",
       " 'envier',\n",
       " 'neurotoxin',\n",
       " 'terzo',\n",
       " 'Og',\n",
       " 'dawnlight',\n",
       " 'proo',\n",
       " 'yowl',\n",
       " 'perciform',\n",
       " 'glee',\n",
       " 'avera',\n",
       " 'concoct',\n",
       " 'amomum',\n",
       " 'dimorph',\n",
       " 'barium',\n",
       " 'shrew',\n",
       " 'propagandist',\n",
       " 'nog',\n",
       " 'fot',\n",
       " 'empt',\n",
       " 'pome',\n",
       " 'wack',\n",
       " 'companionship',\n",
       " 'anadrom',\n",
       " 'sportswear',\n",
       " 'whir',\n",
       " 'cava',\n",
       " 'untrain',\n",
       " 'leaflet',\n",
       " 'chapeau',\n",
       " 'ashen',\n",
       " 'quod',\n",
       " 'intercolumn',\n",
       " 'mokum',\n",
       " 'lungi',\n",
       " 'bacteriostat',\n",
       " 'mestizo',\n",
       " 'ostrich',\n",
       " 'munj',\n",
       " 'scupper',\n",
       " 'liman',\n",
       " 'awlwort',\n",
       " 'kersey',\n",
       " 'pussycat',\n",
       " 'horticulturist',\n",
       " 'stallion',\n",
       " 'brakeman',\n",
       " 'lectual',\n",
       " 'sarsaparilla',\n",
       " 'sitch',\n",
       " 'rind',\n",
       " 'precook',\n",
       " 'positron',\n",
       " 'cantor',\n",
       " 'stanch',\n",
       " 'dhai',\n",
       " 'creed',\n",
       " 'wight',\n",
       " 'zeal',\n",
       " 'circumpolar',\n",
       " 'carbonyl',\n",
       " 'alfresco',\n",
       " 'chaffinch',\n",
       " 'sawdust',\n",
       " 'reobtain',\n",
       " 'looker',\n",
       " 'lifeguard',\n",
       " 'unimodular',\n",
       " 'raceway',\n",
       " 'keratin',\n",
       " 'drool',\n",
       " 'guan',\n",
       " 'becoward',\n",
       " 'beheld',\n",
       " 'abbey',\n",
       " 'cordillera',\n",
       " 'threefold',\n",
       " 'eyelet',\n",
       " 'sordid',\n",
       " 'toeboard',\n",
       " 'smite',\n",
       " 'colder',\n",
       " 'unreal',\n",
       " 'knockdown',\n",
       " 'pagoda',\n",
       " 'flane',\n",
       " 'consonant',\n",
       " 'alchemist',\n",
       " 'rutherford',\n",
       " 'harn',\n",
       " 'furioso',\n",
       " 'dha',\n",
       " 'upo',\n",
       " 'nard',\n",
       " 'nightlong',\n",
       " 'concerto',\n",
       " 'bebar',\n",
       " 'apraxia',\n",
       " 'naiad',\n",
       " 'yachtsman',\n",
       " 'estrogen',\n",
       " 'bastion',\n",
       " 'pouch',\n",
       " 'unio',\n",
       " 'speck',\n",
       " 'saya',\n",
       " 'disinfect',\n",
       " 'teapot',\n",
       " 'creak',\n",
       " 'titi',\n",
       " 'godhead',\n",
       " 'discolor',\n",
       " 'lepidopterist',\n",
       " 'scalpel',\n",
       " 'herewith',\n",
       " 'topper',\n",
       " 'royalist',\n",
       " 'codicil',\n",
       " 'comedown',\n",
       " 'raindrop',\n",
       " 'waar',\n",
       " 'shrivel',\n",
       " 'pock',\n",
       " 'kicker',\n",
       " 'nasi',\n",
       " 'scram',\n",
       " 'tacker',\n",
       " 'apaid',\n",
       " 'cenotaph',\n",
       " 'utch',\n",
       " 'ersatz',\n",
       " 'capuchin',\n",
       " 'quinto',\n",
       " 'barracuda',\n",
       " 'fairyland',\n",
       " 'samson',\n",
       " 'leafless',\n",
       " 'humic',\n",
       " 'marmot',\n",
       " 'brander',\n",
       " 'hatchet',\n",
       " 'tabla',\n",
       " 'toat',\n",
       " 'saccharum',\n",
       " 'yeoman',\n",
       " 'henceforth',\n",
       " 'rondeau',\n",
       " 'mink',\n",
       " 'fingertip',\n",
       " 'tracer',\n",
       " 'quadrivium',\n",
       " 'capybara',\n",
       " 'corbeau',\n",
       " 'mildew',\n",
       " 'fowl',\n",
       " 'soothsay',\n",
       " 'talker',\n",
       " 'tung',\n",
       " 'whirlwind',\n",
       " 'individual',\n",
       " 'maxilla',\n",
       " 'proximo',\n",
       " 'pert',\n",
       " 'brough',\n",
       " 'taxonomist',\n",
       " 'husbandman',\n",
       " 'tempora',\n",
       " 'patrician',\n",
       " 'doff',\n",
       " 'coot',\n",
       " 'dactyl',\n",
       " 'relevant',\n",
       " 'nese',\n",
       " 'megohm',\n",
       " 'sentient',\n",
       " 'limequat',\n",
       " 'branchlet',\n",
       " 'electro',\n",
       " 'damiana',\n",
       " 'sequin',\n",
       " 'bluer',\n",
       " 'malar',\n",
       " 'anta',\n",
       " 'axon',\n",
       " 'dodo',\n",
       " 'shopworn',\n",
       " 'lintel',\n",
       " 'redevelop',\n",
       " 'hoi',\n",
       " 'parson',\n",
       " 'tyke',\n",
       " 'psalm',\n",
       " 'skittish',\n",
       " 'satanist',\n",
       " 'paler',\n",
       " 'desi',\n",
       " 'mola',\n",
       " 'vicar',\n",
       " 'chital',\n",
       " 'kachin',\n",
       " 'hoatzin',\n",
       " 'serum',\n",
       " 'ilex',\n",
       " 'mallard',\n",
       " 'vinifera',\n",
       " 'holler',\n",
       " 'swordfish',\n",
       " 'dimmer',\n",
       " 'adulter',\n",
       " 'tari',\n",
       " 'battel',\n",
       " 'aquavit',\n",
       " 'housecoat',\n",
       " 'decagon',\n",
       " 'vallum',\n",
       " 'armadillo',\n",
       " 'putter',\n",
       " 'tode',\n",
       " 'ephyra',\n",
       " 'spinneret',\n",
       " 'minnow',\n",
       " 'longhair',\n",
       " 'glover',\n",
       " 'sensational',\n",
       " 'natal',\n",
       " 'carbo',\n",
       " 'florid',\n",
       " 'skiff',\n",
       " 'crust',\n",
       " 'ballast',\n",
       " 'rebook',\n",
       " 'bonair',\n",
       " 'chorda',\n",
       " 'comfrey',\n",
       " 'crouton',\n",
       " 'bulkhead',\n",
       " 'moorland',\n",
       " 'stam',\n",
       " 'goi',\n",
       " 'muskeg',\n",
       " 'stench',\n",
       " 'manipular',\n",
       " 'loca',\n",
       " 'corymb',\n",
       " 'downbear',\n",
       " 'bisson',\n",
       " 'chorist',\n",
       " 'octillion',\n",
       " 'smacker',\n",
       " 'egghead',\n",
       " 'vermilion',\n",
       " 'strew',\n",
       " 'persimmon',\n",
       " 'teamwork',\n",
       " 'ascan',\n",
       " 'darter',\n",
       " 'pugilist',\n",
       " 'sweatshop',\n",
       " 'deign',\n",
       " 'stardom',\n",
       " 'coelacanth',\n",
       " 'motorway',\n",
       " 'rata',\n",
       " 'fatherless',\n",
       " 'woodpeck',\n",
       " 'triton',\n",
       " 'repin',\n",
       " 'ament',\n",
       " 'melon',\n",
       " 'zeppelin',\n",
       " 'chard',\n",
       " 'courtier',\n",
       " 'bayamo',\n",
       " 'hangman',\n",
       " 'zag',\n",
       " 'gamecock',\n",
       " 'incognito',\n",
       " 'geneticist',\n",
       " 'oblong',\n",
       " 'thill',\n",
       " 'cyme',\n",
       " 'tua',\n",
       " 'astral',\n",
       " 'prater',\n",
       " 'coauthor',\n",
       " 'yeat',\n",
       " 'raptor',\n",
       " 'dvaita',\n",
       " 'monotheist',\n",
       " 'twite',\n",
       " 'boarder',\n",
       " 'spittoon',\n",
       " 'tweet',\n",
       " 'snifter',\n",
       " 'saccharin',\n",
       " 'trichi',\n",
       " 'testudo',\n",
       " 'daredevil',\n",
       " 'prig',\n",
       " 'headstrong',\n",
       " 'waer',\n",
       " 'rhine',\n",
       " 'kapp',\n",
       " 'grosbeak',\n",
       " 'lum',\n",
       " 'chromo',\n",
       " 'yarrow',\n",
       " 'melodia',\n",
       " 'plainer',\n",
       " 'maggot',\n",
       " 'whereupon',\n",
       " 'kora',\n",
       " 'voivodeship',\n",
       " 'twilight',\n",
       " 'cornea',\n",
       " 'halbert',\n",
       " 'cassinoid',\n",
       " 'manship',\n",
       " 'redcoat',\n",
       " 'hither',\n",
       " 'axial',\n",
       " 'apartheid',\n",
       " 'satchel',\n",
       " 'shim',\n",
       " 'affection',\n",
       " 'wetter',\n",
       " 'pelt',\n",
       " 'tarsi',\n",
       " 'ringlead',\n",
       " 'hypanthium',\n",
       " 'arpent',\n",
       " 'pizzeria',\n",
       " 'gulp',\n",
       " 'empower',\n",
       " 'dool',\n",
       " 'abrupt',\n",
       " 'fiberboard',\n",
       " 'metaphysician',\n",
       " 'nobleman',\n",
       " 'copperhead',\n",
       " 'fontanel',\n",
       " 'nonsectarian',\n",
       " 'mew',\n",
       " 'turbid',\n",
       " 'repen',\n",
       " 'windshield',\n",
       " 'prehuman',\n",
       " 'washbowl',\n",
       " 'auk',\n",
       " 'phono',\n",
       " 'brewster',\n",
       " 'sura',\n",
       " 'survivalist',\n",
       " 'downturn',\n",
       " 'slag',\n",
       " 'nana',\n",
       " 'lora',\n",
       " 'cupola',\n",
       " 'libido',\n",
       " 'henna',\n",
       " 'shredder',\n",
       " 'escheat',\n",
       " 'renal',\n",
       " 'ethel',\n",
       " 'corrigendum',\n",
       " 'muslin',\n",
       " 'reservoir',\n",
       " 'calid',\n",
       " 'tute',\n",
       " 'turndown',\n",
       " 'valerian',\n",
       " 'bigmouth',\n",
       " 'buxom',\n",
       " 'pilar',\n",
       " 'illegal',\n",
       " 'taha',\n",
       " 'terron',\n",
       " 'adolescent',\n",
       " 'pacifist',\n",
       " 'footpad',\n",
       " 'bandstand',\n",
       " 'tired',\n",
       " 'matsu',\n",
       " 'rooker',\n",
       " 'daphnoid',\n",
       " 'acta',\n",
       " 'plover',\n",
       " 'gerfalcon',\n",
       " 'arca',\n",
       " 'stratagem',\n",
       " 'dentata',\n",
       " 'rougher',\n",
       " 'sixer',\n",
       " 'droplet',\n",
       " 'dauber',\n",
       " 'archway',\n",
       " 'workbook',\n",
       " 'cider',\n",
       " 'folia',\n",
       " 'cerium',\n",
       " 'quintic',\n",
       " 'mandala',\n",
       " 'kola',\n",
       " 'missel',\n",
       " 'daresay',\n",
       " 'clitter',\n",
       " 'yokel',\n",
       " 'inheritor',\n",
       " 'sticktight',\n",
       " 'bassist',\n",
       " 'otolith',\n",
       " 'navel',\n",
       " 'bailiwick',\n",
       " 'bight',\n",
       " 'simper',\n",
       " 'chevron',\n",
       " 'mosey',\n",
       " 'uncontrol',\n",
       " 'bairn',\n",
       " 'adieu',\n",
       " 'expressway',\n",
       " 'deme',\n",
       " 'principiant',\n",
       " 'cay',\n",
       " 'multicolor',\n",
       " 'larviform',\n",
       " 'multiplet',\n",
       " 'latchkey',\n",
       " 'fob',\n",
       " 'flaccid',\n",
       " 'marler',\n",
       " 'whitehead',\n",
       " 'scrubland',\n",
       " 'endear',\n",
       " 'waterman',\n",
       " 'hansel',\n",
       " 'fireman',\n",
       " 'planter',\n",
       " 'clove',\n",
       " 'howe',\n",
       " 'charac',\n",
       " 'skulk',\n",
       " 'thacker',\n",
       " 'jann',\n",
       " 'flattish',\n",
       " 'proconsul',\n",
       " 'groper',\n",
       " 'huff',\n",
       " 'cataract',\n",
       " 'whitebark',\n",
       " 'scat',\n",
       " 'treeless',\n",
       " 'proa',\n",
       " 'debar',\n",
       " 'boun',\n",
       " 'bowsprit',\n",
       " 'stagecoach',\n",
       " 'croucher',\n",
       " 'revivalist',\n",
       " 'unsought',\n",
       " 'churr',\n",
       " 'seamanship',\n",
       " 'spay',\n",
       " 'piciform',\n",
       " 'bogard',\n",
       " 'oriel',\n",
       " 'telford',\n",
       " 'microfilm',\n",
       " 'ruru',\n",
       " 'druggist',\n",
       " 'nomad',\n",
       " 'thyroid',\n",
       " 'assistor',\n",
       " 'uitspan',\n",
       " 'wren',\n",
       " 'equestrian',\n",
       " 'dildo',\n",
       " 'overstock',\n",
       " 'hush',\n",
       " 'spooler',\n",
       " 'rehash',\n",
       " 'crayfish',\n",
       " 'scriber',\n",
       " 'mapach',\n",
       " 'ptarmigan',\n",
       " 'sudd',\n",
       " 'daffodil',\n",
       " 'stude',\n",
       " 'tice',\n",
       " 'glycerin',\n",
       " 'fragrant',\n",
       " 'occultist',\n",
       " 'musicianship',\n",
       " 'brill',\n",
       " 'laird',\n",
       " 'kickshaw',\n",
       " 'mukti',\n",
       " 'rancher',\n",
       " 'blanch',\n",
       " 'snowbound',\n",
       " 'carnal',\n",
       " 'covey',\n",
       " 'brushwood',\n",
       " 'bamboo',\n",
       " 'fizz',\n",
       " 'pecker',\n",
       " 'spellbind',\n",
       " 'brickel',\n",
       " 'capulet',\n",
       " 'viga',\n",
       " 'paprika',\n",
       " 'hemlock',\n",
       " 'bettor',\n",
       " 'dere',\n",
       " 'ataman',\n",
       " 'dult',\n",
       " 'flannel',\n",
       " 'rowan',\n",
       " 'brownish',\n",
       " 'beldam',\n",
       " 'entropion',\n",
       " 'beld',\n",
       " 'houseboat',\n",
       " 'draper',\n",
       " 'eyelash',\n",
       " 'norther',\n",
       " 'knob',\n",
       " 'knop',\n",
       " 'savoy',\n",
       " 'betoken',\n",
       " 'attern',\n",
       " 'crowder',\n",
       " 'cuarenta',\n",
       " 'crucifix',\n",
       " 'alderman',\n",
       " 'woodbin',\n",
       " 'labrusca',\n",
       " 'butyl',\n",
       " 'stion',\n",
       " 'extravascular',\n",
       " 'binder',\n",
       " 'exoskeleton',\n",
       " 'declaim',\n",
       " 'tweed',\n",
       " 'rusk',\n",
       " 'peregrin',\n",
       " 'exigent',\n",
       " 'biochemist',\n",
       " 'taum',\n",
       " 'plateau',\n",
       " 'grassland',\n",
       " 'oaken',\n",
       " 'dockyard',\n",
       " 'rattail',\n",
       " 'splendor',\n",
       " 'grout',\n",
       " 'stree',\n",
       " 'causey',\n",
       " 'register',\n",
       " 'bott',\n",
       " 'glottal',\n",
       " 'cockerel',\n",
       " 'japonica',\n",
       " 'cravat',\n",
       " 'chider',\n",
       " 'uncut',\n",
       " 'shyness',\n",
       " 'drier',\n",
       " 'lemur',\n",
       " 'fandom',\n",
       " 'reefer',\n",
       " 'souther',\n",
       " 'microlepidoptera',\n",
       " 'melodrama',\n",
       " 'avast',\n",
       " 'sailboat',\n",
       " 'memento',\n",
       " 'educationalist',\n",
       " 'helminth',\n",
       " 'dorsal',\n",
       " 'gane',\n",
       " 'raja',\n",
       " 'coper',\n",
       " 'dodecahedron',\n",
       " 'heptagon',\n",
       " 'linwood',\n",
       " 'akimbo',\n",
       " 'anteroom',\n",
       " 'sutra',\n",
       " 'talisman',\n",
       " 'nihilist',\n",
       " 'eclat',\n",
       " 'skyway',\n",
       " 'reservist',\n",
       " 'hawthorn',\n",
       " 'simpleton',\n",
       " 'agog',\n",
       " 'orgasm',\n",
       " 'transaction',\n",
       " 'tern',\n",
       " 'caudal',\n",
       " 'grazer',\n",
       " 'ako',\n",
       " 'hereof',\n",
       " 'impost',\n",
       " 'grossen',\n",
       " 'cratch',\n",
       " 'verbena',\n",
       " 'encamp',\n",
       " 'beggar',\n",
       " 'burgh',\n",
       " 'ascidian',\n",
       " 'vagrant',\n",
       " 'milord',\n",
       " 'overzeal',\n",
       " 'barit',\n",
       " 'grue',\n",
       " 'snorkel',\n",
       " 'carillon',\n",
       " 'beseech',\n",
       " 'burgher',\n",
       " 'peal',\n",
       " 'struthioniform',\n",
       " 'kitcat',\n",
       " 'xylitol',\n",
       " 'jarl',\n",
       " 'runt',\n",
       " 'agal',\n",
       " 'teet',\n",
       " 'moveless',\n",
       " 'vermeil',\n",
       " 'lifeless',\n",
       " 'sacrament',\n",
       " 'legate',\n",
       " 'trillium',\n",
       " 'tramway',\n",
       " 'starlight',\n",
       " 'trinitarian',\n",
       " 'peignoir',\n",
       " 'diamagnet',\n",
       " 'plump',\n",
       " 'pupal',\n",
       " 'salicyl',\n",
       " 'hander',\n",
       " 'wadi',\n",
       " 'sweatproof',\n",
       " 'tamarind',\n",
       " 'multiplex',\n",
       " 'pand',\n",
       " 'parakeet',\n",
       " 'hawfinch',\n",
       " 'hydro',\n",
       " 'levant',\n",
       " 'knave',\n",
       " 'tinct',\n",
       " 'matricula',\n",
       " 'wrongdoer',\n",
       " 'mister',\n",
       " 'lifehold',\n",
       " 'nook',\n",
       " 'topmost',\n",
       " 'adjunct',\n",
       " 'injector',\n",
       " 'trill',\n",
       " 'rond',\n",
       " 'thorax',\n",
       " 'bota',\n",
       " 'kinnikinnick',\n",
       " 'northeastward',\n",
       " 'macaroni',\n",
       " 'blackguard',\n",
       " 'brougham',\n",
       " 'tripper',\n",
       " 'supraocular',\n",
       " 'shag',\n",
       " 'earthwork',\n",
       " 'pall',\n",
       " 'feu',\n",
       " 'overfish',\n",
       " 'pinkish',\n",
       " 'sextet',\n",
       " 'paroxysm',\n",
       " 'balderdash',\n",
       " 'heyday',\n",
       " 'broomcorn',\n",
       " 'vanadium',\n",
       " 'mikado',\n",
       " 'lien',\n",
       " 'nightdress',\n",
       " 'kali',\n",
       " 'anuran',\n",
       " 'posey',\n",
       " 'byway',\n",
       " 'carrion',\n",
       " 'daybreak',\n",
       " 'derat',\n",
       " 'arcadian',\n",
       " 'crowbait',\n",
       " 'breadroot',\n",
       " 'nebular',\n",
       " 'blain',\n",
       " 'aneurin',\n",
       " 'depthless',\n",
       " 'Ko',\n",
       " 'fourfold',\n",
       " 'bodkin',\n",
       " 'wallboard',\n",
       " 'basilica',\n",
       " 'steamer',\n",
       " 'airman',\n",
       " 'arhat',\n",
       " 'pernyi',\n",
       " 'scend',\n",
       " 'macrolepidoptera',\n",
       " 'subphylum',\n",
       " 'sixteenth',\n",
       " 'ungloss',\n",
       " 'wem',\n",
       " 'greyhound',\n",
       " 'mysid',\n",
       " 'federalist',\n",
       " 'atlas',\n",
       " 'whither',\n",
       " 'hilt',\n",
       " 'vestal',\n",
       " 'dern',\n",
       " 'pollard',\n",
       " 'westernmost',\n",
       " 'themsel',\n",
       " 'almagest',\n",
       " 'tical',\n",
       " 'camber',\n",
       " 'tripod',\n",
       " 'woodruff',\n",
       " 'formic',\n",
       " 'striction',\n",
       " 'diarrhea',\n",
       " 'retrench',\n",
       " 'yar',\n",
       " 'ponto',\n",
       " 'tup',\n",
       " 'caracol',\n",
       " 'smew',\n",
       " 'casein',\n",
       " 'scallop',\n",
       " 'atman',\n",
       " 'reek',\n",
       " 'bade',\n",
       " 'recti',\n",
       " 'subshrub',\n",
       " 'pur',\n",
       " 'burro',\n",
       " 'freehold',\n",
       " 'carpel',\n",
       " 'statant',\n",
       " 'caesura',\n",
       " 'chanson',\n",
       " 'docent',\n",
       " 'succumb',\n",
       " 'sika',\n",
       " 'motherless',\n",
       " 'southernmost',\n",
       " 'mout',\n",
       " 'ingredient',\n",
       " 'custard',\n",
       " 'tubman',\n",
       " 'crossbreed',\n",
       " 'urea',\n",
       " 'airstrip',\n",
       " 'campground',\n",
       " 'productid',\n",
       " 'olden',\n",
       " 'oclock',\n",
       " 'aluminium',\n",
       " 'augen',\n",
       " 'medievalist',\n",
       " 'neolith',\n",
       " 'waif',\n",
       " 'chartist',\n",
       " 'debutant',\n",
       " 'zapatero',\n",
       " 'majo',\n",
       " 'clincher',\n",
       " 'amour',\n",
       " 'shack',\n",
       " 'retrain',\n",
       " 'caraway',\n",
       " 'thine',\n",
       " 'regrow',\n",
       " 'pumpkin',\n",
       " 'redd',\n",
       " 'charger',\n",
       " 'abigail',\n",
       " 'deerskin',\n",
       " 'nychthemeron',\n",
       " 'holl',\n",
       " 'aught',\n",
       " 'scrambler',\n",
       " 'cebid',\n",
       " 'shanna',\n",
       " 'kiosk',\n",
       " 'unidirect',\n",
       " 'tantra',\n",
       " 'ental',\n",
       " 'assumer',\n",
       " 'deforest',\n",
       " 'ecru',\n",
       " 'rame',\n",
       " 'pagan',\n",
       " 'tablespoon',\n",
       " 'tickler',\n",
       " 'gambol',\n",
       " 'moist',\n",
       " 'limitless',\n",
       " 'hexaploid',\n",
       " 'cero',\n",
       " ...}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import words\n",
    "word_set = set(words.words())\n",
    "word_set.intersection(blacklist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = 0\n",
    "tp = 0\n",
    "fn = 0\n",
    "tn = 0\n",
    "\n",
    "for filename in X_test:\n",
    "    path = os.path.join(DATA_DIR, filename)\n",
    "    if filename in labels:\n",
    "        label = labels[filename]\n",
    "        stems = email_read_util.load(path)\n",
    "        if not stems:\n",
    "            continue\n",
    "        stems_set = set(stems)\n",
    "        if stems_set & blacklist:\n",
    "            if label == 1:\n",
    "                fp = fp + 1\n",
    "            else:\n",
    "                tp = tp + 1\n",
    "        else:\n",
    "            if label == 1:\n",
    "                tn = tn + 1\n",
    "            else:\n",
    "                fn = fn + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table><tr><td>6772</td><td>714</td></tr><tr><td>5835</td><td>7543</td></tr></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import HTML, display\n",
    "conf_matrix = [[tn, fp],\n",
    "               [fn, tp]]\n",
    "display(HTML('<table><tr>{}</tr></table>'.format(\n",
    "    '</tr><tr>'.join('<td>{}</td>'.format(\n",
    "        '</td><td>'.join(str(_) for _ in row)) \n",
    "                     for row in conf_matrix))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table><tr><td>32.5%</td><td>3.4%</td></tr><tr><td>28.0%</td><td>36.2%</td></tr></table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count = tn + tp + fn + fp\n",
    "percent_matrix = [[\"{:.1%}\".format(tn/count), \"{:.1%}\".format(fp/count)],\n",
    "                  [\"{:.1%}\".format(fn/count), \"{:.1%}\".format(tp/count)]]\n",
    "display(HTML('<table><tr>{}</tr></table>'.format(\n",
    "    '</tr><tr>'.join('<td>{}</td>'.format(\n",
    "        '</td><td>'.join(str(_) for _ in row)) \n",
    "                     for row in percent_matrix))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification accuracy: 68.6%\n"
     ]
    }
   ],
   "source": [
    "print(\"Classification accuracy: {}\".format(\"{:.1%}\".format((tp+tn)/count)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
